{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0",
   "metadata": {},
   "source": [
    "# Loading external data\n",
    "\n",
    "This tutorial demonstrates how to load external data into the `ParquetDataCatalog`, and then use this to run a one-shot backtest using a `BacktestNode`.\n",
    "\n",
    "**Warning**:\n",
    "\n",
    "<div style=\"border:1px solid #ffcc00; padding:10px; margin-top:10px; margin-bottom:10px; background-color:#333333; color: #ffcc00;\">\n",
    "Intended to be run on bare metal (not in the jupyterlab docker container)\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "from decimal import Decimal\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from nautilus_trader.backtest.node import BacktestDataConfig\n",
    "from nautilus_trader.backtest.node import BacktestEngineConfig\n",
    "from nautilus_trader.backtest.node import BacktestNode\n",
    "from nautilus_trader.backtest.node import BacktestRunConfig\n",
    "from nautilus_trader.backtest.node import BacktestVenueConfig\n",
    "from nautilus_trader.config import ImportableStrategyConfig\n",
    "from nautilus_trader.core.datetime import dt_to_unix_nanos\n",
    "from nautilus_trader.model import BarType\n",
    "from nautilus_trader.model import QuoteTick\n",
    "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n",
    "from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler\n",
    "from nautilus_trader.test_kit.providers import CSVTickDataLoader\n",
    "from nautilus_trader.test_kit.providers import TestInstrumentProvider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = \"~/Downloads/Data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path(DATA_DIR).expanduser() / \"HISTDATA\"\n",
    "raw_files = list(path.iterdir())\n",
    "assert raw_files, f\"Unable to find any histdata files in directory {path}\"\n",
    "raw_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we just take the first data file found and load into a pandas DataFrame\n",
    "df = CSVTickDataLoader.load(raw_files[0], index_col=0, datetime_format=\"%Y%m%d %H%M%S%f\")\n",
    "df.columns = [\"timestamp\", \"bid_price\", \"ask_price\"]\n",
    "\n",
    "# Process quotes using a wrangler\n",
    "EURUSD = TestInstrumentProvider.default_fx_ccy(\"EUR/USD\")\n",
    "wrangler = QuoteTickDataWrangler(EURUSD)\n",
    "\n",
    "ticks = wrangler.process(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "CATALOG_PATH = os.getcwd() + \"/catalog\"\n",
    "\n",
    "# Clear if it already exists, then create fresh\n",
    "if os.path.exists(CATALOG_PATH):\n",
    "    shutil.rmtree(CATALOG_PATH)\n",
    "os.mkdir(CATALOG_PATH)\n",
    "\n",
    "# Create a catalog instance\n",
    "catalog = ParquetDataCatalog(CATALOG_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write instrument and ticks to catalog\n",
    "catalog.write_data([EURUSD])\n",
    "catalog.write_data(ticks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch all instruments from catalog (as a check)\n",
    "catalog.instruments()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = dt_to_unix_nanos(pd.Timestamp(\"2020-01-03\", tz=\"UTC\"))\n",
    "end =  dt_to_unix_nanos(pd.Timestamp(\"2020-01-04\", tz=\"UTC\"))\n",
    "\n",
    "ticks = catalog.quote_ticks(instrument_ids=[EURUSD.id.value], start=start, end=end)\n",
    "ticks[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "instrument = catalog.instruments()[0]\n",
    "\n",
    "venue_configs = [\n",
    "    BacktestVenueConfig(\n",
    "        name=\"SIM\",\n",
    "        oms_type=\"HEDGING\",\n",
    "        account_type=\"MARGIN\",\n",
    "        base_currency=\"USD\",\n",
    "        starting_balances=[\"1000000 USD\"],\n",
    "    ),\n",
    "]\n",
    "\n",
    "data_configs = [\n",
    "    BacktestDataConfig(\n",
    "        catalog_path=str(catalog.path),\n",
    "        data_cls=QuoteTick,\n",
    "        instrument_id=instrument.id,\n",
    "        start_time=start,\n",
    "        end_time=end,\n",
    "    ),\n",
    "]\n",
    "\n",
    "strategies = [\n",
    "    ImportableStrategyConfig(\n",
    "        strategy_path=\"nautilus_trader.examples.strategies.ema_cross:EMACross\",\n",
    "        config_path=\"nautilus_trader.examples.strategies.ema_cross:EMACrossConfig\",\n",
    "        config={\n",
    "            \"instrument_id\": instrument.id,\n",
    "            \"bar_type\": BarType.from_str(f\"{instrument.id.value}-15-MINUTE-BID-INTERNAL\"),\n",
    "            \"fast_ema_period\": 10,\n",
    "            \"slow_ema_period\": 20,\n",
    "            \"trade_size\": Decimal(1_000_000),\n",
    "        },\n",
    "    ),\n",
    "]\n",
    "\n",
    "config = BacktestRunConfig(\n",
    "    engine=BacktestEngineConfig(strategies=strategies),\n",
    "    data=data_configs,\n",
    "    venues=venue_configs,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "node = BacktestNode(configs=[config])\n",
    "\n",
    "[result] = node.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
